from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from pyquery import PyQuery as pq
from multiprocessing.dummy import Pool
import re,requests,argparse,pymongo,json
import jieba
import jieba.posseg as pseg
import pymysql
import time
import difflib
import sys

def get_parser(): #获取命令行参数
	parser = argparse.ArgumentParser(description='')
	parser.add_argument('-g','--g',type=str,help='')
	parser.add_argument('-c','--c',type=str,help='')
	return parser

def get_comment(): #解析命令行参数
	parser = get_parser()
	args = vars(parser.parse_args())
	if args['g']: #抓取商品列表
		goodstable = 'goodstable' #设置数据表名字
		client = pymongo.MongoClient('localhost',27017)
		dataname = client['goods']
		global table
		table = dataname[goodstable]
		goodsArr = goods2()
		for g in goodsArr:
			print('抓取商品id:',goodsArr[g],g)
			goods_json = search(goodsArr[g],g)
		# goodsArr = goods()
		# for g in goodsArr:
		# 	print('抓取商品id:',g)
		# 	if(goodsArr[g]!=''):
		# 		goods_json = search(goodsArr[g],g)
	if args['c']: #抓取商品评论信息
		goodstable = 'goodstable'
		ranktable = 'goodsrank'
		client = pymongo.MongoClient('localhost',27017)
		
		dataname = client['goods']
		global table2
		table2 = dataname[ranktable]
		table = dataname[goodstable]
		allgoods = [] #获取数据库中的所有商品的itemid和sellerid
		myquery = {}
		for i in table.find(myquery):
			if 'itemid' in i :
				goodsdic = {'itemid': i['itemid'],
				'sellerid': i['sellerid'],'title' : i['title'],'goods_id' : i['goods_id']}
				allgoods.append(goodsdic)
		goods_nums = len(allgoods) #总商品数
		def main(i):
			goods = allgoods[i]
			print('开始抓取评论 {}/{}--->'.format(i, goods_nums))
			crawl_rank(goods['itemid'], goods['sellerid'],goods['title'],goods['goods_id'])

		if __name__ == "__main__": #多线程
			pool = Pool()
			pool.map(main, [i for i in range(len(allgoods))])


def search(goods,g):     #模拟搜索，返回总页数
	try:
		headers = {
			'Referer': 'https://item.taobao.com/item.htm?spm=a219r.lm874.14.173.2d324edc7BaCKr&id=591671671551&ns=1&abbucket=9',
			'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4238.2 Safari/537.36',
			'cookie':"cna=8aBcF5hOblwCAXyB64oUgOws; miid=190114509729061816; hng=CN%7Czh-CN%7CCNY%7C156; thw=cn; sgcookie=E100m1h8BD3m9oOfXVb0T8pI4PVKpXFGUthAbe8sXI5eCYrRBwima0nxs563LLfJlofBkhEDk7xsZJkr%2B%2FNTKgK0CA%3D%3D; uc3=nk2=FPjanhObXgVTAtU%3D&id2=UNGfFnqOEicUYw%3D%3D&vt3=F8dCufbH7iB1ZhX5HKw%3D&lg2=UIHiLt3xD8xYTw%3D%3D; lgc=wangheng669; uc4=nk4=0%40FnNat5P%2BI92ai3gAEnLaggYdMlZdnQ%3D%3D&id4=0%40Ugbnn7NL6kTLsd5nYB8V%2B9PBj6fV; tracknick=wangheng669; _cc_=URm48syIZQ%3D%3D; enc=CktROg2HSfnctP%2FM0iGfqdmd65fOnH1CNv3Aj1wMdJ2oJ6%2FoPHUrwHnVgWuoZUP%2Flizdz14jgoTxseCOkxuSTg%3D%3D; mt=ci=1_1; xlly_s=1; _samesite_flag_=true; cookie2=123e90c90019a83a21bac211526db805; t=48948ba988b1dd3de9e398873c43d3c7; _tb_token_=ee17eb93e7d87; _m_h5_tk=9e5b6a5681da89baa99f77ff5898dde4_1600140678400; _m_h5_tk_enc=daf37e615b63112ad1feb2099fe8bb86; v=0; uc1=cookie14=Uoe0bUnPZMtNtQ%3D%3D; lLtC1_=1; x5sec=7b22726174656d616e616765723b32223a223065303464323537353538376536356232633665653936396631333437353434434b764367507346454e5864364f2b57763653584d773d3d227d; tfstk=cv81BP196R21kqn47CGF3w_4jzbOZEF5Gc6M1hT3YFVINBA1i_UPF6PMm-Ud261..; l=eBgdj-74OzxI81VABO5aFurza77OzIRb8sPzaNbMiInca6C5N3TnCNQ4D7sXldtjgtCX2etyWNfbrR3v8szdgETNJqHzH13jfxvO.; isg=BL-_SqVL3JY4BNizNOz3LiHnTpNJpBNGqLt5XlGMi26MYN7iWXVsllk2ojCeOOu-"
		}
		url = 'https://s.taobao.com/search?q='+goods
		web = requests.get(url, headers=headers)
		goods_match = re.search(r'g_page_config = (.*?)}};', web.text)
		goods_str = goods_match.group(1) + '}}'
		goods_json = json.loads(goods_str)

		if goods_json['mods']['itemlist']['status'] == 'hide' :
			print('提取数据失败！',g)
			errgoods = {
				'goods_id':g
			}
			save_to_mongo(errgoods,table,0)
			# sys.exit(0)
			return True
		goods_items = goods_json['mods']['itemlist']['data']['auctions']
		return get_products(goods_items,goods,g)
	except BaseException:  #超时则重新调用
		print('抓取失败--请求次数过多')
def get_products(goodList,keyword,g):
	goods_list = []
	for goods_item in goodList:
		goods_item.setdefault('view_sales','0人付款')
		goods = {'title': goods_item['raw_title'],
                     'price': goods_item['view_price'],
                     'location': goods_item['item_loc'],
                     'sales': goods_item['view_sales'],
                     'comment_url': goods_item['comment_url'],
					 'sellerid': goods_item['user_id'],
					 'itemid': goods_item['nid'],
					 'resemble': get_equal_rate(goods_item['raw_title'], keyword),
					 'goods_id' : g,
					}
		if goods['sales'] != '0人付款':
		# save_to_mongo(goods,table,0)
			goods_list.append(goods)
	goods_list.sort(key=takeSecond,reverse=True)
	print(goods_list[0]['title'])
	save_to_mongo(goods_list[0],table,0)

def crawl_rank(itemid, sellerid,goods_name,goods_id):    #itemid和sellerid是抓取商品信息得到的，用来构造评论信息的url
	url = 'https://rate.taobao.com/feedRateList.htm?'
	query_params = {
		'auctionNumId': itemid,
		'userNumId': sellerid,
		'currentPageNum': '1',
		'pageSize': '12',
		'rateType': '',
		'orderType': 'sort_weight',
		'attribute': '',
		'sku': '',
		'hasSku': 'false',
		'folded': '1',
		'ua': '134#xb7IeXXwXGEAnTfLJXuZdJtDO8giig/Vp2imRasPwn65PACvqszG0E4Hs3O5hwg3EFHCUk703ZVOyJYFep2dUnxXYfoAZNFi/tUwarx/tint+Tq6IgL7oJJqqH8uiNv1QWLFCqpAZtwtV9T8Aqn5orLwqTpqqygFCOsPVLwIwZ4sNBP092I/evkyxfpjxsAT+XUi+2UDrW5JZXKMv77xGWzeeNGK1NKTBDMlnlfZFBzd84GGaxMwQ+Jc7/5jluaXiRoIqDyIlOUS5xVkaO1sjtzah2aiZDSnNjAmnoBenfRlkasi93H8wbp0FUJmFJrQwWW9zqHZy1W4T6FxxVkSwyz3wzEdffpcnvww3UJJLNnSYSW9HPSxYT0MHklv1vutijR1P4Cj2tFxKGvUsrFxoyfwUFFcuNVu9z0JZbxshBpFE1Rurt5FmzSmC8jmjzzpky8d7eoNNNbZIxyDZymBvSXi9jRzCLgYWxX4jtIIHfn7LHiQWlLeydDMywG2p/DdXJM9ooXXABvevVtc1VH51NlFKAedrzz0uyeZju7K6gTKCNgnZTNnjLfQAE78lRLlkH2cFfvNbCpCtcQfggkUsq60WKqrMygXRa5371UO7kqUV/jpaPEfU/wwXEW+FmCP+zgtXFVb4MvmIAQAYEB0Kn/Ez6LuiCQl4HszPkkZPGApwNJfassnGaVbeFQJ3iRcIEA5w13HrxXQQuPWLVCQVXPAlD/MGwllHrMyLa99p9hZkSV2NFyd0unch8eUyP4q1n5ls8BxL3mzyv9FwNdgfZZ125h=',
		'_ksTS': '1600237467381_1651',
		'callback': 'jsonp_tbcrate_reviews_list',
	}
	headers = {
		'Referer': 'https://item.taobao.com/item.htm?spm=a219r.lm874.14.173.2d324edc7BaCKr&id=591671671551&ns=1&abbucket=9',
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4238.2 Safari/537.36',
		'cookie':"cna=8aBcF5hOblwCAXyB64oUgOws; miid=190114509729061816; hng=CN%7Czh-CN%7CCNY%7C156; thw=cn; sgcookie=E100m1h8BD3m9oOfXVb0T8pI4PVKpXFGUthAbe8sXI5eCYrRBwima0nxs563LLfJlofBkhEDk7xsZJkr%2B%2FNTKgK0CA%3D%3D; uc3=nk2=FPjanhObXgVTAtU%3D&id2=UNGfFnqOEicUYw%3D%3D&vt3=F8dCufbH7iB1ZhX5HKw%3D&lg2=UIHiLt3xD8xYTw%3D%3D; lgc=wangheng669; uc4=nk4=0%40FnNat5P%2BI92ai3gAEnLaggYdMlZdnQ%3D%3D&id4=0%40Ugbnn7NL6kTLsd5nYB8V%2B9PBj6fV; tracknick=wangheng669; _cc_=URm48syIZQ%3D%3D; enc=CktROg2HSfnctP%2FM0iGfqdmd65fOnH1CNv3Aj1wMdJ2oJ6%2FoPHUrwHnVgWuoZUP%2Flizdz14jgoTxseCOkxuSTg%3D%3D; mt=ci=1_1; xlly_s=1; _samesite_flag_=true; cookie2=123e90c90019a83a21bac211526db805; t=48948ba988b1dd3de9e398873c43d3c7; _tb_token_=ee17eb93e7d87; _m_h5_tk=9e5b6a5681da89baa99f77ff5898dde4_1600140678400; _m_h5_tk_enc=daf37e615b63112ad1feb2099fe8bb86; v=0; uc1=cookie14=Uoe0bUnPZMtNtQ%3D%3D; lLtC1_=1; x5sec=7b22726174656d616e616765723b32223a223065303464323537353538376536356232633665653936396631333437353434434b764367507346454e5864364f2b57763653584d773d3d227d; tfstk=cv81BP196R21kqn47CGF3w_4jzbOZEF5Gc6M1hT3YFVINBA1i_UPF6PMm-Ud261..; l=eBgdj-74OzxI81VABO5aFurza77OzIRb8sPzaNbMiInca6C5N3TnCNQ4D7sXldtjgtCX2etyWNfbrR3v8szdgETNJqHzH13jfxvO.; isg=BL-_SqVL3JY4BNizNOz3LiHnTpNJpBNGqLt5XlGMi26MYN7iWXVsllk2ojCeOOu-"
	}
	maxpage = get_page(url, query_params,headers) 
	for page in range(1, maxpage + 1):
		try:
			response = requests.get(url=url, headers=headers, params=query_params).text
			rex=re.compile(r'\w+[(]{1}(.*)[)]{1}')
			content=rex.findall(response)[0]
			data=json.loads(content)
			items = data['comments']     # 遍历items，保存到mongodb
			for item in items:
				goodsrate = {
					'date': item['date'],
					'title': item['user']['nick'],
					'content': item['content'],
					'goods_name': goods_name,
					'goods_id' : goods_id
				}
				save_to_mongo(goodsrate,table2,page_number=-1)
		except:
			continue

def get_page(url, query_params,headers):
	try:
		response = requests.get(url=url, headers=headers, params=query_params).text
		rex=re.compile(r'\w+[(]{1}(.*)[)]{1}')
		content=rex.findall(response)[0]
		data=json.loads(content)
		return data['maxPage']
	except:
		return get_page(url, query_params,headers)

def save_to_mongo(product,mongotable,page_number):
	try:
		if mongotable.insert_one(product):
			if page_number != -1: 
				print('保存数据 --->'.format(page_number), product['title'])
			if page_number == -1:
				print('保存评论 --->',product['title'])
	except Exception:
		print('save MongoDB failed', product)

def goods2():
	goodstable = 'goods' #设置数据表名字
	client = pymongo.MongoClient('localhost',27017)
	dataname = client['goods']
	table = dataname[goodstable]
	result = table.find()
	goodsArr = {}
	for g in result:
		words = pseg.cut(g['goods_name'],use_paddle=True) #paddle模式
		s = ''
		for word, flag in words:
			if(flag == 'n' or flag == 'nr' or flag == 'nz' or flag == 'j' or flag == 'ns' or flag == 't' or flag == 'v' or flag == 'z' or flag == 'a'):
				if(word != '卓越' and word != '特警' and word != '陆军'):
					s += word
		goodsArr[g['goods_id']] = s
	return goodsArr
def goods():
	# 建立数据库连接
	conn=pymysql.connect(
		host='127.0.0.1',
		port=3306,
		user='root',
		password='root',
		db='dscmall',
		charset='utf8'
	)
	# 获取游标
	cursor=conn.cursor(pymysql.cursors.DictCursor)
	# 执行sql语句
	sql =  'select * from dsc_goods where goods_id > 13804'
	rows=cursor.execute(sql)  # 返回结果是受影响的行
	goods = cursor.fetchall()
	goodsArr = {}
	for g in goods:
		words = pseg.cut(g['goods_name'],use_paddle=True) #paddle模式
		s = ''
		for word, flag in words:
			if(flag == 'n' or flag == 'nr' or flag == 'nz'):
				s += word
		goodsArr[g['goods_id']] = s
	# 关闭游标
	cursor.close()
	# 关闭连接
	conn.close()
	return goodsArr
# 字符串相似度对比
def get_equal_rate(str1, str2):
    return difflib.SequenceMatcher(None, str1, str2).quick_ratio()
def takeSecond(elem):
    return elem['resemble']
if __name__ == "__main__":

	get_comment()
